#!/usr/bin/env bash

################################################################################
#                                                                              #
# Shell-script for running Dalton                                              #
#                                                                              #
# Version 1.0 Feb. 1997 by T. Enevoldsen, Odense University                    #
# Version 1.1 spring 2000 by K.Ruud and H.J.Aa.Jensen                          #
# Version 2.1 Feb. 2005 by K.Ruud, H.J.Aa.Jensen, and P. Salek                 #
# Version 2011, 2014, 2016, 2020 by H.J.Aa.Jensen, with help from others       #
#                                                                              #
################################################################################

mpi_enabled=@ENABLE_MPI@

dalton="dalton"
DALTON="DALTON"

# radovan: this is to figure out the location of this script
# http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
    SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
    SOURCE="$(readlink "$SOURCE")"
    # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
    [[ $SOURCE != /* ]] && SOURCE="$SCRIPT_DIR/$SOURCE"
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"


## If your computer ressource uses modules, then you can make sure the right modules are loaded
## by uncommenting and adapting the two lines starting with #module following this comment.
## The sample "module load" below provides an example of what might be needed for a "./setup --int64 --mpi" build
## in order to load an integer*8 mpi instead of a default integer*4 mpi.
## Of course the same modules should be loaded as those used for the build.
## NOTE! The module statements must come after the definition of SCRIPT_DIR above, otherwise that may not work.
## /Comment added Oct. 2016, hjaaj.
#module purge
#module load python intel openmpi-i8

DALTON_EXECUTABLE=$SCRIPT_DIR/$dalton.x

which mpirun > /dev/null # check if mpirun exists, if not, then assume mpiexec
if [ $? -eq 0 ]; then
   MPIEXEC=mpirun
else
   MPIEXEC=mpiexec
fi
INSTALL_BASDIR=$SCRIPT_DIR/basis
#
paramlist="$*"
# default extensions
OUTEXT="out"
DALEXT="dal"
MOLEXT="mol"
POTEXT="pot"
PCMEXT="pcm"

GET_FILES=''
PUT_FILES=''
dalfil=''
molfil=''
potfil=''
pcmfil=''
SSH=ssh
SCP=scp

# DALTON_NUM_MPI_PROCS is deprecated, print a warning if it is used
if [ -n "$DALTON_NUM_MPI_PROCS" ]; then
    echo "WARNING: DALTON_NUM_MPI_PROCS is deprecated and will be removed in future releases, use DALTON_LAUNCHER instead"
fi

#default values for options
optd=0
optD=1
optf=0
optn=${DALTON_NUM_MPI_PROCS:-1} # default value must be 1 for running Dalton in sequential mode with OpenMPI.
opto=0
optpg=0
opt_cpexe=0
opt_nobackup=0
opt_noarch=0
opt_noappend=0
optx=0
WRKDIR=`pwd`
#
# Define usage message
#
usage (){
        echo
        echo "Usage: $0 [options] dalinp{.$DALEXT} [molinp{.$MOLEXT} [potinp{.$POTEXT}] [pcmsolver{.$PCMEXT}]]"
        echo
        echo 'Options:'
        echo ' -w dir           : change job directory to dir (default:' `pwd` ')'
        echo ' -b dir           : prepend dir to directory list for basis set searches'
        echo '                    (job directory and dalton basis library are included automatically)'
        echo ' -o file          : redirect output from program to job directory with "file" as file name'
        echo ' -ow              : redirect output from program to job directory with standard file name'
        echo ' -dal file        : the '$dalton' input file'
        echo ' -mol file        : the molecule input file'
        echo ' -pot file        : the potential input file (for .QM3, .QMMM and .PELIB)'
        echo ' -pcm file        : the pcm input file'
        echo ' -ext log         : change default output extension from ".out" to ".log"'
        echo ' -nobackup        : do not backup files, simply overwrite outputs'
        echo ' -f dal_mol[_pot] : extract dal_mol[_pot].tar.gz archive from WRKDIR into'
        echo '                    DALTON_TMPDIR before calculation starts'
        echo ' -noarch          : do not create tar.gz archive'
        echo " -t dir           : set scratch directory DALTON_TMPDIR; this script will append"
        echo "                    '/DALTON_scratch_$USER' to the path unless the path"
        echo "                    contains 'DALTON_scratch' or you explicitly set -noappend"
        echo ' -d               : delete job scratch directory before calculation starts'
        echo ' -D               : do not delete job scratch directory after calculation stops'
        echo ' -noappend        : do not append anything to the scratch directory; be careful'
        echo '                    with this option since by default scratch is wiped after calculation'
        echo ' -get "file1 ..." : get files back from DALTON_TMPDIR after calculation stops'
        echo ' -put "file1 ..." : put files to DALTON_TMPDIR before calculation starts'
        echo ' -omp num         : set the number of OpenMP threads. Note that Dalton is not OpenMP'
        echo '                    parallelized, however, this option can be used with e.g. threaded blas as MKL'
     if [ $mpi_enabled = "ON" ] ; then  # MPI parallel version
        echo ' -N num |-np num  : use num MPI processes (defaults to 1, illegal if DALTON_LAUNCHER specified)'
     fi
        echo ' -cpexe           : copy '$dalton'.x to DALTON_TMPDIR before execution, either to global scratch'
        echo '                    (if DALTON_USE_GLOBAL_SCRATCH is set) or to local scratch on all nodes'
     if [ $mpi_enabled = "ON" ] ; then  # MPI parallel version
        echo ' -rsh             : use rsh/rcp for communication with MPI nodes (default: ssh/scp)'
        echo ' -nodelist "node1 ..." : set nodelist DALTON_NODELIST, '$dalton'.x will be copied to'
        echo '                    DALTON_TMPDIR on each node unless DALTON_USE_GLOBAL_SCRATCH is'
        echo '                    defined (the script uses PBS_NODEFILE or SLURM_NODELIST if available)'
     fi
        echo ' -x dalmol1 dalmol2 : calculate NEXAFS spectrum from ground and core hole states'
        echo " -exe exec        : change the executable from default ($DALTON_EXECUTABLE) to exec"
        echo ' -pg              : do profiling with gprof'
        echo ' -gb mem          : set dalton max usable work memory to mem Gigabytes (mem integer)'
        echo ' -mb mem          : set dalton max usable work memory to mem Megabytes (mem integer)'
#       echo ' -mw mem          : set dalton max usable work memory to mem double precision words'
     if [ $mpi_enabled = "ON" ] ; then  # MPI parallel version
        echo ' -ngb mem         : set node max usable work memory to mem Gigabytes (mem integer)'
        echo ' -nmb mem         : set node max usable work memory to mem Megabytes (mem integer)'
#       echo ' -nw mem          : set node max usable work memory to mem double precision words'
     fi
        echo
        echo "Environment variables understood by the $0 script:"
        echo '    DALTON_TMPDIR             : scratch directory'
     if [ $mpi_enabled = "ON" ] ; then  # MPI parallel version
        echo '    DALTON_USE_GLOBAL_SCRATCH : use global scratch directory, do not copy any files to worker nodes'
        echo '    DALTON_NODELIST           : list of nodes, '$dalton'.x will be copied to DALTON_TMPDIR on each node'
        echo '                                unless DALTON_USE_GLOBAL_SCRATCH is defined'
     fi
     if [ $mpi_enabled = "ON" ] ; then  # MPI parallel version
        echo '    DALTON_LAUNCHER           : launcher for the '$dalton'.x binary (if defined, -N flag not allowed)'
     else
        echo '    DALTON_LAUNCHER           : launcher for the '$dalton'.x binary'
     fi
         }
#
# Define backup function
#
backup () {
  if [ $opt_nobackup -eq 0 ]; then
      if [ "$1" = "-v" ]; then ekko=1; shift
      else ekko=0;
      fi
      for i in $* ; do
         if [ -f "$i" ]; then
            for j in 6 5 4 3 2 1 0 ; do
               jp=`expr $j + 1`
               if [ -f "$i.$j" ]; then
                  [ $ekko -eq 1 ] && echo "Backup: renaming $i.$j to $i.$jp"
                  mv -f "$i.$j" "$i.$jp"
               fi
            done
            if [ $ekko -eq 1 ]; then echo "Backup: renaming $i to $i.0"; fi
            mv -f "$i" "$i.0"
         fi
      done
  fi
}

#
# Interpret input
#
while [ -n "`echo $1 | grep '^-'`" ]; do
    case $1 in
      -b ) BASDIR="$2${BASDIR:+:${BASDIR}}"; shift;;
      -d ) optd=1;;
      -D ) optD=0;;
      -nobackup ) opt_nobackup=1;;
      -noarch ) opt_noarch=1;;
      -cpexe ) opt_cpexe=1;;
      -noappend ) opt_noappend=1;;
      -pg ) optpg=1;;
      -nodelist ) DALTON_NODELIST=$2; shift;;
      -ext ) OUTEXT=$2; shift;;
      -get ) GET_FILES="$GET_FILES $2"; shift;;
      -put ) PUT_FILES="$PUT_FILES $2"; shift;;
      -rsh ) SCP=rcp; SSH=rsh;;
      -f ) optf=$2; shift;;
      -N|-np ) optn=$2; shift;;
      -omp ) export OMP_NUM_THREADS=$2; shift;;
      -ow ) opto=2;;
      -o ) OUTFIL=$2; opto=1; shift;;
      -t ) DALTON_TMPDIR=$2; shift;;
      -w ) WRKDIR=$2; shift;;
      -exe ) DALTON_EXECUTABLE=$2; shift;;
      -dal ) dalfil=$2; shift;;
      -mol ) molfil=$2; shift;;
      -pot ) potfil=$2; shift;;
      -pcm ) pcmfil=$2; shift;;
      -m|-mw ) WRKMEM=$2; export WRKMEM; shift;;
      -M|-mb ) WRKMEM=`expr $2 \* 128000`; export WRKMEM; shift;;
       -gb ) WRKMEM=`expr $2 \* 128000 \* 1024`; export WRKMEM; shift;;
       -nw ) NODE_WRKMEM=$2; export NODE_WRKMEM; shift;;
       -nb|-nmb ) NODE_WRKMEM=`expr $2 "*" 128000`; export NODE_WRKMEM; shift;;
       -ngb ) NODE_WRKMEM=`expr $2 \* 128000 \* 1024`; export NODE_WRKMEM; shift;;
       -v|--version ) echo "version: @DALTON_VERSION@"; exit 0;;
       -x ) optx=1; gs=$2; ch=$3; shift;shift;;
        * ) echo "Unrecognized option: $1"; usage; exit 1;;
   esac
   shift
done

#          OUTFIL and WRKDIR are read from environment inside dalton.x
export OUTFIL
export WRKDIR

# always search for basis sets in job work directory
# (before standard dalton basis library in INSTALL_BASDIR, appended below)
BASDIR="${BASDIR:+${BASDIR}:}$WRKDIR"
# append default basis set directory
len_INSTALL_BASDIR=`echo $INSTALL_BASDIR | wc -c`
if [ $len_INSTALL_BASDIR -gt 1 ]; then
   # do not count newline char
   BASDIR="$BASDIR:$INSTALL_BASDIR"
fi
export BASDIR

# -- identify input files and construct output file name

# check for correct input
if [ -z "$dalfil" ] && [ -z "$1" ]; then
    echo 'Fatal error: Input file[s] not specified'
    echo "Your command was: $0 $paramlist"
    usage
    exit 1
fi

if [ -n "$dalfil" ]; then
    # dal given as -dal <filename>
    # remove dal-ending
    dalfil=`echo $dalfil | sed "s/\.$DALEXT//"`
    if [ -n "$molfil" ]; then
        molfil=`echo $molfil | sed "s/\.$MOLEXT//"`
	    # otherwise molecule assumed defined inside dalfil
    fi
    if [ -n "$potfil" ]; then
        potfil=`echo $potfil | sed "s/\.$POTEXT//"`
    fi
    if [ -n "$pcmfil" ]; then
        pcmfil=`echo $pcmfil | sed "s/\.$PCMEXT//"`
    fi
elif [ -n "$1" ]; then
    # input files given as arguments in the end of the dalton command
    dalfil=`echo $1 | sed "s/\.$DALEXT//"`
    if [ -z "$2" ]; then
	    # only one argument given
	    if [ -r ${dalfil}.$MOLEXT ]; then
	        # molfil named the same as dalfil
	        molfil=$dalfil
	        # otherwise molecule assumed defined inside dalfil
    	fi
        if [ -r ${dalfil}.$POTEXT ]; then
            potfil=$dalfil
        fi
        if [ -r ${dalfil}.$PCMEXT ]; then
            pcmfil=$dalfil
        fi
    else
        molfil=`echo $2 | sed "s/\.$MOLEXT//"`
    	if [ -n "$3" ]; then
            tmpfil=`echo $3 | sed "s/\.$POTEXT//" | sed "s/\.$PCMEXT//"`
            if [ -r ${tmpfil}.$POTEXT ]; then
                potfil=$tmpfil
            elif [ -r ${tmpfil}.$PCMEXT ]; then
                pcmfil=$tmpfil
            else
                echo "ERROR: positional parameter $3 not recognized as .$POTEXT or .$PCMEXT file"
                exit 1
            fi
        fi
    fi
fi

if [ -z "$potfil" ] && [ -z "$pcmfil" ]; then
    if [ "$dalfil" = "$molfil" ] || [ -z "$molfil" ]; then
	    outnam=$dalfil
    else
	    outnam=${dalfil}_${molfil}
    fi
    input_files="input files: $dalfil.$DALEXT and $molfil.$MOLEXT"
elif [ -n "$potfil" ]; then
    if [ "$dalfil" = "$molfil" ] && [ "$dalfil" = "$potfil" ] ; then
	    outnam=$dalfil
    else
	    outnam=${dalfil}_${molfil}_${potfil}
    fi
    input_files="input files: $dalfil.$DALEXT, $molfil.$MOLEXT, and $potfil.$POTEXT"
elif [ -n "$pcmfil" ]; then
    if [ "$dalfil" = "$molfil" ] && [ "$dalfil" = "$pcmfil" ] ; then
	    outnam=$dalfil
    else
	    outnam=${dalfil}_${molfil}_${pcmfil}
    fi
    input_files="input files: $dalfil.$DALEXT, $molfil.$MOLEXT, and $pcmfil.$PCMEXT"
fi

outnam=$(echo $outnam | sed 's|/|-|g')

if [ $opto -eq 2 ]; then   # -ow input option
   OUTFIL=$outnam.$OUTEXT
   export OUTFIL
fi
if [ -n "$OUTFIL" ]; then
if [ -s  $OUTFIL -a -r $OUTFIL ]; then
   backup -v $OUTFIL
fi
fi

# -- check if DALTON_TMPDIR defined; if not provide a guess


if [ -z "$DALTON_TMPDIR" ]; then  # not set as environment variable and not set with "-t dir" option
    scratch_dir_candidates="$SCRATCH /global/work/$USER /scratch/$USER /work/$USER /work /scratch /scr /temp /tmp"
    for dir in $scratch_dir_candidates; do
        if [ -d "$dir" ]; then
            DALTON_TMPDIR=$dir
            echo "INFO: DALTON_TMPDIR not defined by user."
            echo "INFO: Guessing $dir is an appropriate scratch directory, and DALTON_TMPDIR is set to this directory"
            break
        fi
    done

    if [ -z "$DALTON_TMPDIR" ]; then
        # DALTON_TMPDIR is still not set, better stop here
        echo "ERROR: searched for suitable scratch directories among"
        echo "       $scratch_dir_candidates"
        echo "       but none of them exists."
        echo "       Please either set the DALTON_TMPDIR environment variable"
        echo "       or use -t to set the scratch directory."
        exit 1
    fi
fi

if [ $opt_noappend -eq 0 ]; then
    if [[ "$DALTON_TMPDIR" != */ ]]; then
        # if there is no final slash, we append it
        DALTON_TMPDIR=${DALTON_TMPDIR}/
    fi
    # now we append DALTON_scratch_$USER
    # better safe than sorry
    DALTON_TMPDIR=${DALTON_TMPDIR}DALTON_scratch_$USER
fi
#
export DALTON_TMPDIR
#
if [ ! -d "$DALTON_TMPDIR" ]; then
    mkdir -p $DALTON_TMPDIR
fi
if [ ! -d "$DALTON_TMPDIR" ]; then
    # this means that user has no permission to create scratch dir, stop
    echo "ERROR: you have no permission to create $DALTON_TMPDIR"
    exit 1
fi

# now we know that the directory exists
# we will now try whether user has write permission to this directory
file=$DALTON_TMPDIR/test_whether_dalton_can_create_this_file.$$.tmp
if [ -f $file ]; then
    rm $file &> /dev/null
fi
touch $file &> /dev/null
if [ -f $file ]; then
    rm $file &> /dev/null
else
    rm $file &> /dev/null
    echo "ERROR: you have no write permission to scratch directory $DALTON_TMPDIR"
    exit 1
fi

if [ $opt_noappend -eq 1 ]; then
    SCRATCHDIR=$DALTON_TMPDIR
else
    # append pid to scratch directory to avoid multiple tests running in the same directory
    SCRATCHDIR=$DALTON_TMPDIR/${outnam}_$$
fi

echo
echo "   ******************************************* "
echo "   **** OUTPUT FROM "$DALTON" SHELL SCRIPT"
echo "   ******************************************* "
echo;
echo "   DALTON release @DALTON_VERSION@"
echo;
echo "   Invocation: $0 $paramlist"
echo;
echo "   `date`"
echo;
echo "   Calculation: $outnam  ($input_files)"
echo "   PID        : $$"
echo "   Input dir  : $WRKDIR"
echo "   Scratch dir: $SCRATCHDIR"
echo;

dalton_exitcode=0
if [ -d "$SCRATCHDIR" ]; then
   if [ $optd -eq 1 ] ; then
      rm -rf $SCRATCHDIR/*
   fi
else
   mkdir -p $SCRATCHDIR
fi
cd $SCRATCHDIR

# edh: Small script to get updated RST duing run. 
# Can be handy if the run is likely to be interrupted e.g. by tight time-limits...
# Experimental; if it turns out handy it can be made into an input option :-) 
#echo "#!/bin/bash                            "  > dal_backup.sh
#echo "sleep 30s                              " >> dal_backup.sh
#echo "while (pgrep "dalton.x" > /dev/null);do" >> dal_backup.sh
#echo "sleep 10m                              " >> dal_backup.sh 
#echo "echo "copy RST file from scratch.."    " >> dal_backup.sh
#echo "cp $SCRATCHDIR/SIRIUS.RST $WRKDIR      " >> dal_backup.sh
#echo "done                                   " >> dal_backup.sh
#[[ -f dal_backup.sh ]] && echo "File exist" || echo "File does not exist"
#/bin/bash $SCRATCHDIR/dal_backup.sh & 

if [ -s $WRKDIR/$dalfil.$DALEXT -a -r $WRKDIR/$dalfil.$DALEXT ]; then
   cp $WRKDIR/$dalfil.$DALEXT $DALTON.INP
else
   echo "$WRKDIR/$dalfil.$DALEXT does not exist or is not readable"
   dalton_exitcode=1
fi
if [ -s $WRKDIR/$molfil.$MOLEXT -a -r $WRKDIR/$molfil.$MOLEXT ]; then
   cp $WRKDIR/$molfil.$MOLEXT MOLECULE.INP
else
   echo "$WRKDIR/$molfil.$MOLEXT does not exist or is not readable"
   if [ $dalton_exitcode = 0 ]; then
      echo " => assuming user has written MOLECULE input in dalton input file"
   fi
fi
if [ "x$potfil" != "x" ]; then
   if [ -s $WRKDIR/$potfil.$POTEXT -a -r $WRKDIR/$potfil.$POTEXT ]; then
      if [[ $(file --mime-type -b "$WRKDIR/$potfil.$POTEXT") == application/x-hdf ]]; then
         cp $WRKDIR/$potfil.$POTEXT pelib_temp.h5
         h5unjam -i pelib_temp.h5 -o standard.h5 -u POTENTIAL.INP
      else
         cp $WRKDIR/$potfil.$POTEXT POTENTIAL.INP
      fi
   else
      echo "$WRKDIR/$potfil.$POTEXT does not exist or is not readable"
      dalton_exitcode=1
   fi
fi

if [ "x$pcmfil" != "x" ]; then
   if [ -s $WRKDIR/$pcmfil.$PCMEXT -a -r $WRKDIR/$pcmfil.$PCMEXT ]; then
      cp $WRKDIR/$pcmfil.$PCMEXT pcmsolver.inp
      python @PCMSolver_PARSE_DIR@/pcmsolver.py pcmsolver.inp
   else
      echo "$WRKDIR/$pcmfil.$PCMEXT for PCM does not exist or is not readable"
      dalton_exitcode=1
   fi
fi

# put files listed under -put to scratch directory
if [ "x$PUT_FILES" != "x" ]; then
   for file in $PUT_FILES; do
      echo copying $file to scratch directory
      cp $WRKDIR/$file $file
   done
fi

if [ "$optf" != "0" ] ; then
   if [ -s $WRKDIR/$optf.tar.gz -a -r $WRKDIR/$optf.tar.gz ] ; then
      echo "====================="
      echo "The '-f' option is active, now unpacking $optf.tar.gz into:"
      tar -z -x -v -f $WRKDIR/$optf.tar.gz
      echo "====================="
   elif [ -s $WRKDIR/$optf -a -r $WRKDIR/$optf ] ; then
#     hjaaj: at least GNU tar finds out autmatically if tar, tar.gz or
#     tar.bzip2 -- hopefully your tar program also does that!
      echo "====================="
      echo "The '-f' option is active, now unpacking $optf into:"
      tar -x -v -f $WRKDIR/$optf
      echo "====================="
   else
      echo "$WRKDIR/$optf.tar.gz does not exist or is not readable"
      dalton_exitcode=1
   fi
fi

if [ $dalton_exitcode = 1 ]; then
   exit 1
fi

if [ $opt_cpexe -eq 1 ] ; then
   cp $DALTON_EXECUTABLE $SCRATCHDIR/dalton.x
   if [ $optn -gt 1 -a -z "$DALTON_USE_GLOBAL_SCRATCH" ] ; then
      if [ -n "$DALTON_NODELIST" ] ; then
         NODELIST=$(echo "$DALTON_NODELIST" | tr ' ' '\n' | sort -u | tr '\n' ' ')
      elif [ -n "$PBS_NODEFILE" ] ; then
         NODELIST=`sort -u $PBS_NODEFILE`
      elif [ -n "$SLURM_NODELIST" ] ; then
         NODELIST=$(scontrol show hostname $SLURM_NODELIST)
      else
         echo " The '-cpexe' option is active and more than one MPI process requested,"
         echo " but no nodelist was found. A list of nodes can be provided using the"
         echo " '-nodelist \"node1 ...\"' option or by setting DALTON_NODELIST=\"node1 ...\"."
         echo " Attempt to continue using executable in scratch on the local node."
         NODELIST=$HOSTNAME
      fi
      for node in $NODELIST ; do
         if [ ${node} == ${HOSTNAME%\.local} ] ; then
            continue
         else
            $SSH ${node} "mkdir -p $SCRATCHDIR"
            $SCP $DALTON_EXECUTABLE ${node}:$SCRATCHDIR/dalton.x
         fi
      done
   fi
   DALTON_EXECUTABLE=$SCRATCHDIR/dalton.x
fi

if ! [ -s "$DALTON_EXECUTABLE" -a -x "$DALTON_EXECUTABLE" ]; then
   echo "ERROR: $DALTON_EXECUTABLE does not exist or is not executable"
   exit 1
fi

if [ -n "$DALTON_USE_GLOBAL_SCRATCH" ] ; then
   echo ' DALTON_USE_GLOBAL_SCRATCH is defined, so Dalton will use global scratch dir.'
fi

if [ $mpi_enabled = "ON" -a $optn -gt 1 ] ; then  # MPI parallel
   if [ -z "$OMP_NUM_THREADS" ]; then
      export OMP_NUM_THREADS=1
      echo
      echo ' INFO : OMP_NUM_THREADS set to 1 because it was not defined by user'
      echo ' INFO : and all the available cores are probably used by MPI.'
      echo
   fi
fi

if [ -n "$DALTON_LAUNCHER" ] ; then
    if [ $optn -gt 1 ] ; then
       echo; echo  "*** JOB SPECIFICATION ERROR:"
       echo "    DALTON_LAUNCHER was defined as: $DALTON_LAUNCHER"
       echo "    Definition of DALTON_LAUNCHER is not compatible with -N/-np flag nor definition of DALTON_NUM_MPI_PROCS"
       echo "--> Include instead number of MPI processes in your definition of DALTON_LAUNCHER"; echo
       exit 1
    fi
    DALTON_EXE="$DALTON_LAUNCHER $DALTON_EXECUTABLE"
elif [ $mpi_enabled = "ON" ] ; then
    is_OpenMPI=`$MPIEXEC --version | grep -i -c 'open.*mpi'`
    if [ $is_OpenMPI -gt 0 ] ; then
       # For openMPI: if --bind-to none is not added, then e.g. "ctest -j 10" for "-np 1" will run all 10 jobs on CPU core 0
       # and the remaining CPU cores will idle ... observed 03-May-2021)
       DALTON_EXE="$MPIEXEC --bind-to none -np $optn $DALTON_EXECUTABLE"
    else
       DALTON_EXE="$MPIEXEC -np $optn $DALTON_EXECUTABLE"
    fi
else
    if [ $optn -gt 1 ] ; then
       echo; echo  "*** JOB SPECIFICATION ERROR:"
       echo "    $optn MPI nodes requested for a non-MPI enabled dalton executable."; echo
       exit 1
    fi
    DALTON_EXE=$DALTON_EXECUTABLE
fi

$DALTON_EXE
dalton_exitcode=$?
if [ $dalton_exitcode -ne 0 ] ; then
   echo; echo  "SERIOUS ERROR:"
   echo "$DALTON_EXE finished with non-zero exit code: $dalton_exitcode"
   echo; echo "File list in scratch directory:";echo
   ls -lt
   du -h
   # No "exit $dalton_exitcode" here, because we want to save output and
   # tar-file for restart possibilities !!!!!!
fi

if [ $optpg -eq 1 ] ; then
    gprof $DALTON_EXECUTABLE gmon.out > PROFILE
fi

if [ "$optx" != "0" ] ; then
   if [ -s $WRKDIR/$gs.tar.gz -a -r $WRKDIR/$gs.tar.gz ] ; then
      echo "The '-x' option is active, now unpacking $gs.tar.gz into:"
      gunzip < $WRKDIR/$gs.tar.gz | tar xvf - SIRIFC
      if [ -s SIRIFC -a -r SIRIFC ] ; then
         mv -v SIRIFC SIRIFC1
      else
         echo $WRKDIR/$gs.tar.gz does not contain interface file SIRIFC
         dalton_exitcode=1
      fi
   else
      echo "$WRKDIR/$gs.tar.gz does not exist or is not readable"
      dalton_exitcode=1
   fi
   if [ -s $WRKDIR/$ch.tar.gz -a -r $WRKDIR/$ch.tar.gz ] ; then
      echo "The '-x' option is active, now unpacking $ch.tar.gz into:"
      gunzip < $WRKDIR/$ch.tar.gz | tar xvf - SIRIFC
      if [ -s SIRIFC -a -r SIRIFC ] ; then
         mv -v SIRIFC SIRIFC2
      else
         echo $WRKDIR/$ch.tar.gz does not contain interface file SIRIFC
         dalton_exitcode=1
      fi
   else
      echo "$WRKDIR/$ch.tar.gz does not exist or is not readable"
      dalton_exitcode=1
   fi
fi

# list of files which will be saved in $outnam.tar.gz
# for subsequent calculations
filelist="DALTON.STAT SIRIUS.RST RESULTS.RSP RESPONSE.C8 RSPVEC ABSVECS SIRIFC* "
filelist="$filelist DALTON.NCA DALTON.MOL DALTON.ORB DALTON.IRC DALTON.BAS DALTON.TRJ DALTON.CM DALTON.HES DALTON.MOPUN DALTON.WLK"
filelist="$filelist molden.inp DALTON.PROP LUCITA_CVECS.a LUCITA_CVECS.b LUCITA_CVECS.c LUCITA_CVECS.d LUCITA_CVECS.e LUCITA_CVECS.f LUCITA_CVECS.g LUCITA_CVECS.h"
filelist="$filelist gv.off FCIDUMP UNIT1 UNIT2 rsp_tensor_human final_geometry.xyz *.cube"
pcmfiles="cavity.off PCM_matrix PCM_mep_asc PEDRA.OUT *.npy *.npz *.vtk molec_dyadic.dat"
pelib_files="pelib_electrostatics.bin pelib_induced_moments.bin pelib_induced_charges.bin pelib_response_matrix.bin pelib_nuclear_potential.bin pelib_nuclear_field.bin pelib_multipole_potential.bin pelib_multipole_field.bin"
filelist="$filelist $pcmfiles $pelib_files $GET_FILES"

if [ -s $DALTON.OUT.n0001 -a -r $DALTON.OUT.n0001 ]; then
   tarfilelist="$DALTON.OUT.n*"
else
   tarfilelist=
fi

for i in $filelist ; do
   if [ -s $i -a -r $i ] ; then
      tarfilelist=$tarfilelist" "$i
   fi
done

if [ $opt_noarch -eq 0 ]; then
    if [ -s "first.wrl" -a -r "first.wrl" ] ; then
       tar cf - $tarfilelist *wrl | gzip -9 > $outnam.tar.gz
    else
       tar cf - $tarfilelist | gzip -9 > $outnam.tar.gz
    fi
fi

if [ -s $outnam.tar.gz ] ; then
   backup -v $WRKDIR/$outnam.tar.gz
   cp $outnam.tar.gz $WRKDIR
   echo "$outnam.tar.gz has been copied to $WRKDIR"
else
    echo "$outnam.tar.gz has not been created and has thus not been copied to $WRKDIR"
fi

# copy files listed under -get back from scratch directory
if [ "x$GET_FILES" != "x" ]; then
   echo Copying files requested with -get back from scratch directory
   for file in $GET_FILES; do
       cp -v -p $file $WRKDIR/$outnam.$file
   done
fi

if [ $opto -ne 0 ] ; then
   if [ -s $WRKDIR/$OUTFIL -a -r $WRKDIR/$OUTFIL ]; then
      echo "Output is in $WRKDIR/$OUTFIL as requested in input."
   else
      echo "$WRKDIR/$OUTFIL has not been created from the present run"
      if [ $optD -eq 1 ] ; then
         echo "$SCRATCHDIR is therefore not deleted by this script."
      fi
      exit 2
   fi
else
   if [ -s $DALTON.OUT -a -r $DALTON.OUT ]; then
      OUTFIL=$outnam.$OUTEXT
      backup -v $WRKDIR/$OUTFIL
      cp $DALTON.OUT $WRKDIR/$OUTFIL
      [ -f PROFILE ] && cp -f PROFILE $WRKDIR/$outnam.prof.$OUTEXT
   else
      echo "$DALTON.OUT has not been created from the present run."
      if [ $optD -eq 1 ] ; then
         echo "$SCRATCHDIR is therefore not deleted by this script."
      fi
      echo "List of created files in $SCRATCHDIR :"
      ls -sltr
      exit 2
   fi
fi

cd $WRKDIR

# check if error exit, warnings, or info messages

NERR=`grep -c ERROR $OUTFIL`
if [ $dalton_exitcode -ne 0 -o $NERR -gt 0 ] ; then
   echo "----------------------------------------------------------"
   echo "WARNING for ERROR  ($DALTON program stopped with exit code $dalton_exitcode)"
   if [ $optD -eq 1 ] ; then
      echo "- THEREFORE scratch directory $SCRATCHDIR is not deleted"
      echo "  by this script so you may restart the calculation, if possible"
      echo "  ( $SCRATCHDIR may be deleted by your queueing system )"
      echo "- REMEMBER to delete $SCRATCHDIR yourself !!!"; echo
      optD=0
   fi
   [ $NERR -gt 0 ] && ( echo "Output from 'grep -n ERROR' :"; grep -n ERROR $OUTFIL )
fi

nwrn=`grep -c -i WARNING $OUTFIL`
if [ $nwrn -gt 0 ] ; then
   echo "----------------------------------------------------------"
   echo "$nwrn WARNINGS have been issued by $DALTON,"
   echo "output from 'grep -n -i WARNING'  (max 10 warnings):"
   grep -n -i "WARNING" $OUTFIL | head -10
fi

ninfo=`grep -c '[^"]INFO' $OUTFIL`
if [ $ninfo -gt 0 ] ; then
   echo "----------------------------------------------------------"
   echo "$ninfo informational messages have been issued by $DALTON,"
   echo "output from 'grep -n INFO'  (max 10 lines):"
   grep -n INFO $OUTFIL | grep -v 'Check output, result, and error' | head -10
fi

if [ $optD -eq 1 ] ; then
   rm -rf $SCRATCHDIR
fi

echo; echo "   `date`"
echo "   ***************************************** "

if [ $dalton_exitcode -ne 0 ]; then
   exit $dalton_exitcode
fi
# -- end of the dalton script --
